const Crawler = require("crawler");
const config = require('./config');
const http = require('http');
const path = require('path');
const cheerioTableparser = require('cheerio-tableparser');


const shopsFileName = path.join(process.cwd(), 'shops_data.json');





var c = new Crawler({
    maxConnections: 10,
    jQuery: true,
    // This will be called for each crawled page
    callback: function (error, res, done) {
        if (error) {
            console.log(error);
        } else {
            console.log(res.statusCode);
            //console.log(res.body);
            if (res.statusCode === 200) {
                console.log('callback');
                var $ = res.$;

                console.log($('#comment').children().length);

                console.log($('.sub-title').text());

                var commentCount = $('.item current').children('span').text();

                console.log(commentCount);


                $('.content').children('span').children('a').each((i, element) => {

                    console.log($(element).text());
                });

            }
        }
        done();
    }
});
http.get('http://www.dianping.com/mylist/ajax/shoprank?cityId=4&shopType=10&rankType=popscore&categoryId=0', (res) => {
    if (res.statusCode === 200) {
        let rawData = '';
        res.on('data', (chunk) => rawData += chunk);
        res.on('end', () => {
            try {
                const parsedData = JSON.parse(rawData);

                //循环得出所有店铺地址
                // parsedData.shopBeans.forEach(function (shop) {

                //     let shopUrl = `http://www.dianping.com/shop/${shop.shopId}`;

                //     console.log(`shopUrl: ${shopUrl}`);


                    c.queue('http://www.dianping.com/shop/91009319');


                // });



            } catch (e) {
                console.log(e.message);
            }
        });
    }
});

c.on('schedule', function (options) {
    options.rateLimit = 5000;
    options.userAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36';
    //console.log('schedule event');
});

c.on('request', (options) => {
    //console.log('request event', JSON.stringify(options));
});

c.on('drain', () => {
    console.log('queue is empty!');
});

